package com.cgy.utils.base.test.spider;

import com.cgy.utils.spider.adaptor.WebMagicAdaptor;
import com.cgy.utils.spider.callback.AbstractCallBack;
import com.cgy.utils.spider.config.WebMagicConfig;
import com.cgy.utils.spider.element.CssElement;
import com.cgy.utils.spider.element.TemplateElement;
import com.cgy.utils.spider.resolver.WebMagicResolver;

import java.io.File;
import java.util.Map;


/**
 * 测试
 * @author CH
 *
 */

public class WebMagicExample {

	public static void main(String[] args) {
		//创建自定义模板
		TemplateElement te = (TemplateElement) new TemplateElement().template(new File("F:/spider1.xml")).build();
		//创建css模板
		CssElement cssElement = (CssElement) new CssElement(true).build();
		//配置器
		WebMagicConfig webMagicConfig = WebMagicConfig.create();
		//解析器
		webMagicConfig.setResolver(new WebMagicResolver());
		//线程数
		webMagicConfig.setThread(10);
		//节点
		webMagicConfig.elements(te, cssElement);
		//设置回调
		webMagicConfig.callback(new AbstractCallBack() {
			@Override
			public void callback(Map<String, Object> pages) {
				System.out.println(pages);
			}
		});
		//html下载器
		//webMagicConfig.setDownloader(new HtmlUnitDownloader());
		//适配器
		WebMagicAdaptor webMagicAdaptor = WebMagicAdaptor.create();
		//装载
		webMagicAdaptor.setConfig(webMagicConfig).runAsync("https://www.douyu.com/65000");
	}
}

